# -*- coding: utf-8 -*-

import os
from dataclasses import dataclass
from typing import List, Tuple

# 全局调试模式开关
DEBUG = False

@dataclass
class SpiderConfig:
    """爬虫配置"""
    name: str
    site_name: str
    allowed_domains: List[str]
    concurrent_requests: int
    download_delay: float

@dataclass
class GlobalConfig:
    """全局配置"""
    output_dir: str = "outputs"
    max_file_size: int = 100 * 1024 * 1024  # 100MB
    max_records_per_file: int = 10000
    download_delay: float = 1.0
    download_timeout: int = 180
    download_delay_range: Tuple[float, float] = (0.1, 0.5)
    debug: bool = DEBUG  # 使用全局DEBUG变量

def create_default_config():
    """创建默认配置"""
    return {
        "global": {
            "output_dir": "outputs",
            "max_file_size": 100 * 1024 * 1024,  # 100MB
            "max_records_per_file": 10000,
            "download_delay": 1.0,
            "download_timeout": 180,
            "download_delay_range": [0.1, 0.5],
            "debug": DEBUG
        },
        "spiders": [
            {
                "name": "beijingfire",
                "site_name": "beijingfire_com",
                "allowed_domains": [
                    "beijingfire.com",
                    "www.beijingfire.com"
                ],
                "concurrent_requests": 4,
                "download_delay": 2.0
            },
            {
                "name": "londonfire",
                "site_name": "london_fire_gov_uk",
                "allowed_domains": [
                    "london-fire.gov.uk",
                    "www.london-fire.gov.uk"
                ],
                "concurrent_requests": 4,
                "download_delay": 2.0
            },
            {
                "name": "cccf",
                "site_name": "cccf_com_cn",
                "allowed_domains": [
                    "cccf.com.cn",
                    "www.cccf.com.cn"
                ],
                "concurrent_requests": 4,
                "download_delay": 2.0
            },
            {
                "name": "nist",
                "site_name": "nist_gov",
                "allowed_domains": [
                    "nist.gov",
                    "www.nist.gov",
                    "pages.nist.gov"
                ],
                "concurrent_requests": 4,
                "download_delay": 2.0
            },
            {
                "name": "iaff",
                "site_name": "iaff_org",
                "allowed_domains": [
                    "iaff.org",
                    "www.iaff.org"
                ],
                "concurrent_requests": 4,
                "download_delay": 2.0
            },
            {
                "name": "sfpe",
                "site_name": "sfpe_org",
                "allowed_domains": [
                    "sfpe.org",
                    "www.sfpe.org"
                ],
                "concurrent_requests": 4,
                "download_delay": 2.0
            },
            {
                "name": "nfpa",
                "site_name": "nfpa_org",
                "allowed_domains": [
                    "nfpa.org",
                    "www.nfpa.org"
                ],
                "concurrent_requests": 4,
                "download_delay": 2.0
            }
        ]
    }

def get_scrapy_settings():
    """获取Scrapy设置"""
    return {
        'BOT_NAME': 'fire_control_spider',
        'SPIDER_MODULES': [
            'fire_control_spider.sites.beijingfire',
            'fire_control_spider.sites.londonfire',
            'fire_control_spider.sites.cccf',
            'fire_control_spider.sites.nist',
            'fire_control_spider.sites.iaff',
            'fire_control_spider.sites.sfpe',
            'fire_control_spider.sites.nfpa'
        ],
        'NEWSPIDER_MODULE': 'fire_control_spider.sites',
        'ROBOTSTXT_OBEY': False,
        'CONCURRENT_REQUESTS': 20,
        'CONCURRENT_REQUESTS_PER_DOMAIN': 10,
        'DOWNLOAD_DELAY': 0.05,
        'DOWNLOAD_TIMEOUT': 180,
        'COOKIES_ENABLED': False,
        'TELNETCONSOLE_ENABLED': False,
        'ITEM_PIPELINES': {
            'fire_control_spider.pipelines.ValidationPipeline': 300,
            'fire_control_spider.pipelines.DuplicatesPipeline': 400,
            'fire_control_spider.pipelines.async_download.AsyncMediaDownloadPipeline': 250,  # 异步下载管道
            'fire_control_spider.pipelines.JsonlWriterPipeline': 600,
            # 'fire_control_spider.pipelines.MediaDownloadPipeline': 500,  # 禁用默认的同步下载管道
        },
        'DOWNLOADER_MIDDLEWARES': {
            'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
        },
        'LOG_LEVEL': 'CRITICAL',
        'LOG_ENABLED': False,
    }